import requests
import random
from fake_useragent import UserAgent
from lxml import etree
# from pymysql import connect
#
# one.connect = connect(
#     user="root",
#     password="123456",
#     host="127.0.0.1",
#     port=3306,
#     database="demotext",
#     charset="utf-8"
# )
user_agent_list = []
for i in range(20):
    user_agent_list.append(UserAgent().random)

for num in range(1,3):
    headers = {"user-agent": random.choice(user_agent_list)}
    print("正在爬取第{}页信息".format(num))
    url = "http://www.hnbitebi.com/hlist-7-{}.html".format(num)
    html=requests.get(url,headers=headers).text
    # print(html)
    parse_html = etree.HTML(html)
    r_listlink = parse_html.xpath('//ul[@class="list2"]/li//a/@href')
    #print(r_list)
    r_listtitle=parse_html.xpath('//ul[@class="list2"]/li//a/text()')

    #解析二级页面
    for link,j in zip(r_listlink,r_listtitle):
        print(j)
        print(link)
        html = requests.get(link, headers=headers).text
        parse_html = etree.HTML(html)
        r_listin = parse_html.xpath('//div[@class="con"]/div/p/text()')
        # r_listin=r_listin.pop(0)
        for i in r_listin:
            print(i,end='')
        print("============================")
